{
 "metadata": {
  "name": "Louvain Test"
 }, 
 "name": "Louvain Test", 
 "nbformat": 2, 
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown", 
     "source": "Before testing, ensure your `ZK_LIST` property is set correctly at line 8 within distributed-louvain-modularity/louvain.py."
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cd distributed-louvain-modularity/example", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "/home/vagrant/distributed-louvain-modularity/example"
      }
     ], 
     "prompt_number": 1
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "./run_example.sh", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "clear out old example dir.."
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "Deleted /tmp/louvain-giraph-example\nloading example data to /tmp/louvain-giraph-example"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "running the full louvain pipeline\n\nrunning full louvain on /tmp/louvain-giraph-example/input\nsaving output to: /tmp/louvain-giraph-example/output\n\nRemoving ouput dir"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "Deleted /tmp/louvain-giraph-example/output"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "\n\nrunning giraph phase /tmp/louvain-giraph-example/input -> /tmp/louvain-giraph-example/output/giraph_1\n"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "rm: `/tmp/louvain-giraph-example/output/giraph_1': No such file or directory"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:41 INFO utils.ConfigurationUtils: No edge input format specified. Ensure your InputFormat does not require one.\n14/01/26 03:24:41 INFO utils.ConfigurationUtils: Setting custom argument [giraph.vertex.input.dir] to [/tmp/louvain-giraph-example/input] in GiraphConfiguration"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "\n14/01/26 03:24:41 INFO utils.ConfigurationUtils: Setting custom argument [ mapreduce.task.timeout] to [10800000] in GiraphConfiguration\n14/01/26 03:24:41 INFO utils.ConfigurationUtils: Setting custom argument [actual.Q.aggregators] to [1] in GiraphConfiguration\n14/01/26 03:24:41 INFO utils.ConfigurationUtils: Setting custom argument [minimum.progress] to [2000] in GiraphConfiguration\n14/01/26 03:24:41 INFO utils.ConfigurationUtils: Setting custom argument [progress.tries] to [1] in GiraphConfiguration"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:41 INFO job.GiraphJob: run: Since checkpointing is disabled (default), do not allow any task retries (setting mapred.map.max.attempts = 0, old value = 4)"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:41 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same."
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:43 INFO mapred.JobClient: Running job: job_201401260304_0002"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:44 INFO mapred.JobClient:  map 0% reduce 0%"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:24:59 INFO mapred.JobClient:  map 50% reduce 0%"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:25:00 INFO mapred.JobClient:  map 100% reduce 0%"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:25:02 INFO mapred.JobClient: Job complete: job_201401260304_0002\n14/01/26 03:25:02 INFO mapred.JobClient: Counters: 34\n14/01/26 03:25:02 INFO mapred.JobClient:   File System Counters\n14/01/26 03:25:02 INFO mapred.JobClient:     FILE: Number of bytes read=0\n14/01/26 03:25:02 INFO mapred.JobClient:     FILE: Number of bytes written=410326\n14/01/26 03:25:02 INFO mapred.JobClient:     FILE: Number of read operations=0\n14/01/26 03:25:02 INFO mapred.JobClient:     FILE: Number of large read operations=0\n14/01/26 03:25:02 INFO mapred.JobClient:     FILE: Number of write operations=0\n14/01/26 03:25:02 INFO mapred.JobClient:     HDFS: Number of bytes read=432\n14/01/26 03:25:02 INFO mapred.JobClient:     HDFS: Number of bytes written=134\n14/01/26 03:25:02 INFO mapred.JobClient:     HDFS: Number of read operations=10\n14/01/26 03:25:02 INFO mapred.JobClient:     HDFS: Number of large read operations=0\n14/01/26 03:25:02 INFO mapred.JobClient:     HDFS: Number of write operations=4\n14/01/26 03:25:02 INFO mapred.JobClient:   Job Counters \n14/01/26 03:25:02 INFO mapred.JobClient:     Launched map tasks=2\n14/01/26 03:25:02 INFO mapred.JobClient:     Total time spent by all maps in occupied slots (ms)=26203\n14/01/26 03:25:02 INFO mapred.JobClient:     Total time spent by all reduces in occupied slots (ms)=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Total time spent by all maps waiting after reserving slots (ms)=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Total time spent by all reduces waiting after reserving slots (ms)=0"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "14/01/26 03:25:02 INFO mapred.JobClient:   Map-Reduce Framework\n14/01/26 03:25:02 INFO mapred.JobClient:     Map input records=2\n14/01/26 03:25:02 INFO mapred.JobClient:     Map output records=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Input split bytes=88\n14/01/26 03:25:02 INFO mapred.JobClient:     Spilled Records=0\n14/01/26 03:25:02 INFO mapred.JobClient:     CPU time spent (ms)=2690\n14/01/26 03:25:02 INFO mapred.JobClient:     Physical memory (bytes) snapshot=269602816\n14/01/26 03:25:02 INFO mapred.JobClient:     Virtual memory (bytes) snapshot=3734609920\n14/01/26 03:25:02 INFO mapred.JobClient:     Total committed heap usage (bytes)=253100032\n14/01/26 03:25:02 INFO mapred.JobClient:   Giraph Stats\n14/01/26 03:25:02 INFO mapred.JobClient:     Aggregate edges=11\n14/01/26 03:25:02 INFO mapred.JobClient:     Aggregate finished vertices=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Aggregate vertices=9\n14/01/26 03:25:02 INFO mapred.JobClient:     Current master task partition=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Current workers=1\n14/01/26 03:25:02 INFO mapred.JobClient:     Last checkpointed superstep=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Sent messages=0\n14/01/26 03:25:02 INFO mapred.JobClient:     Superstep=14\n14/01/26 03:25:02 INFO mapred.JobClient:   Giraph Timers\n14/01/26 03:25:02 INFO mapred.JobClient:     Setup (milliseconds)=1167\n14/01/26 03:25:02 INFO mapred.JobClient:     Shutdown (milliseconds)=232\n14/01/26 03:25:02 INFO mapred.JobClient:     Total (milliseconds)=2741"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "giraph exit status: 0 time (sec): 27.4048581123"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "Found 1 items\n-rw-r--r--   1 vagrant supergroup         18 2014-01-26 03:24 /tmp/louvain-giraph-example/output/_COMPLETE\ngiraph times 27.4048581123\ntotal giraph time: 27.4048581123\nmapreduce times \ntotal mapreduce time 0\nTotal time: 27.4048581123"
      }
     ], 
     "prompt_number": 2
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": "hadoop fs -get /tmp/louvain-giraph-example/output .", 
     "language": "python", 
     "outputs": [], 
     "prompt_number": 3
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cd output", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "/home/vagrant/distributed-louvain-modularity/example/output"
      }, 
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": ""
      }
     ], 
     "prompt_number": 4
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "ls -la", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "total 20\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[0m\u001b[01;34m.\u001b[0m/\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[01;34m..\u001b[0m/\n-rwxr-xr-x 1 vagrant vagrant   18 Jan 26 03:25 \u001b[01;32m_COMPLETE\u001b[0m*\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[01;34mgiraph_1\u001b[0m/\n-rwxr-xr-x 1 vagrant vagrant   18 Jan 26 03:25 \u001b[01;32m_q_1\u001b[0m*"
      }
     ], 
     "prompt_number": 5
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cat _COMPLETE", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "0.4134948096885813"
      }
     ], 
     "prompt_number": 6
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cd giraph_1/", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "/home/vagrant/distributed-louvain-modularity/example/output/giraph_1"
      }
     ], 
     "prompt_number": 7
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "ls -la", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "total 16\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[0m\u001b[01;34m.\u001b[0m/\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[01;34m..\u001b[0m/\ndrwxrwxr-x 3 vagrant vagrant 4096 Jan 26 03:25 \u001b[01;34m_logs\u001b[0m/\n-rwxr-xr-x 1 vagrant vagrant   98 Jan 26 03:25 \u001b[01;32mpart-m-00001\u001b[0m*\n-rwxr-xr-x 1 vagrant vagrant    0 Jan 26 03:25 \u001b[01;32m_SUCCESS\u001b[0m*"
      }
     ], 
     "prompt_number": 8
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cat part-m-00001", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "4\t3\t0\t3:4\n2\t3\t0\t3:4\n9\t8\t0\t8:3\n5\t3\t0\t3:4\n6\t8\t0\t3:1,8:3\n3\t3\t0\t3:4\n7\t8\t0\t8:3\n1\t3\t0\t3:4,8:1\n8\t8\t0\t8:3"
      }
     ], 
     "prompt_number": 9
    }, 
    {
     "cell_type": "code", 
     "collapsed": false, 
     "input": "cat ../../small.tsv", 
     "language": "python", 
     "outputs": [
      {
       "output_type": "stream", 
       "stream": "stdout", 
       "text": "1\t0\t2:1,3:1,4:1,5:1,6:1\n2\t0\t1:1,3:1,4:1,5:1\n3\t0\t1:1,2:1,4:1,5:1\n4\t0\t1:1,2:1,3:1,5:1\n5\t0\t1:1,2:1,3:1,4:1\n6\t0\t1:1,7:1,8:1,9:1\n7\t0\t6:1,8:1,9:1\n8\t0\t6:1,7:1,9:1\n9\t0\t6:1,7:1,8:1"
      }
     ], 
     "prompt_number": 10
    }, 
    {
     "cell_type": "code", 
     "collapsed": true, 
     "input": "", 
     "language": "python", 
     "outputs": []
    }
   ]
  }
 ]
}